if __name__ == '__main__':
    pass

from html.parser import HTMLParser
import re

class NewLFCHistoryParser(HTMLParser):
    
    def config(self, outfile):
        self.strict = True
        self.out = outfile
    
    def handle_data(self, data):
        if len(data.rstrip()) >  0:
            self.out.write(data + ' @ ')

class goal_type:
    ordinary = 0
    penalty = 1
    own_goal = 2        

class Parse_Recent_Games():    
    def __init__ (self):
        self.read_ids('inc/games')#('data/games2')
        self.read_stadiums('data/stadium.txt')
        self.read_players('data/players.txt')
        self.read_edited_players('data/comp_players_edited.txt')
        self.read_own_scorers('data/own_scorers.txt')
        self.read_opposition('data/opposition.txt')
        self.read_referees('data/referees.txt')
        self.read_tournaments()
        
    def read_files(self, path, dump):
        import os
        
        dump = open(dump, 'w', encoding = 'utf-8')
        
        for infile in os.listdir(path):
            print ("current file is:", path+infile)
            id = infile[infile.find('_')+1:infile.find('.')]
            dump.write(id + ' @ ') 
                    
            parser = NewLFCHistoryParser()
            parser.config(dump)
                    
            data = open(path+infile, 'r', encoding = 'utf-8').read()
            start = data.find("<h2 class=\"page-name\">Report</h2>") + len("<h2 class=\"page-name\">Report</h2>")
            end = data[start:].find("<h3>Notes</h3>") + start
            
            parser.feed(data[start:end])
            
            dump.write('\n')        
    
    def process_recent(self, file):
        refs = []
        all_dicts = []
        
        out_file = open('data/dump_jan17.txt', 'w', encoding = 'utf-8')
        
        with open(file, 'r', encoding = 'utf-8') as recent_file:
            for line in recent_file.readlines():
                parts = line.split('@')
                self.dict = {}
                self.dict['goals'] = []
                self.dict['red cards'] = []
                #print (parts)
                
                pos = 0
                self.parse_id(parts[0].strip())
                self.parse_score(parts[1].strip()) 
                pos = parts.index(' Game date: ') + 1
                self.parse_date(parts[pos].strip())
                pos = parts.index(' Stadium: ') + 1
                self.parse_stadium(parts[pos].strip())
                pos = parts.index(' Competition: ') + 1
                self.parse_tournament(parts[pos].strip())
                pos = parts.index(' Attendance: ') + 1
                self.parse_attendance(parts[pos].strip())
                                
                try:
                    pos = parts.index(' Referee: ') + 1
                    self.parse_referee(parts[pos])
                except ValueError:
                    ''' No referee info '''
                    self.parse_referee('Unknown')


                pos = parts.index(' Starting lineup ') + 1
                self.parse_line_up(parts[pos:pos+33])
                pos += 33
    
                action = ''
                
                while True:
                    function = self.get_section(parts[pos])
                                        
                    if function == 'exit':
                        break
                    elif function == 'continue':
                        shift = action(parts[pos:])
                    else:
                        action = function 
                        pos += 1
                        shift = action(parts[pos:])
                        
                        
                    #assert shift > 0, parts[pos:]
                    pos += shift
                    
                    if pos >= len(parts):
                        break 
                                             
                #print ('-- ', self.dict)#, '\n')
                out_file.write(str(self.dict)+'\n')
                #self.generate_goals_stats_sql('') #(out_file)
                all_dicts.append(self.dict)
                
                #break
        out_file.close()
        self.full_dict = all_dicts
        
    def check_goalscorers(self, all_dicts):
        marksmen = {}
        for item in all_dicts:
            for [scorer, type, minute] in item['goals']:
                    
                if type == goal_type.own_goal:
                    continue
                    
                if scorer in marksmen:
                    marksmen[scorer] += 1
                else:
                    marksmen[scorer] = 1

        
        of_stat = []

        for line in open('data/temp', 'r', encoding='utf-8').readlines():
            items = line.split('\t')
            of_stat.append([items[0], items[1], items[2].strip()]) 


        count = 0
        for key, scored in zip(sorted(marksmen, key=marksmen.get, reverse=True), sorted(marksmen.values(), reverse=True)):
            for name in self.players.keys():
                if self.players[name] == key:
                    player = name
                    break
            else:
                assert False, key
    
            print (name, scored)
                
            if not int(of_stat[count][2]) == scored: 
                print (count, name, scored)
                print (of_stat[count], '\n')
    
            count += 1              
    
    def parse_void(self, text):
        print ('pv:', text)

    
    def get_section(self, text):
        functions = {' Subs '          : self.parse_sub,\
                    ' Goals '          : self.parse_goal,\
                    ' Substitutions '  : self.parse_substitutions,\
                    ' Red Cards '      : self.parse_red_cards,\
                    ' Notes '          : self.parse_notes,\
                    ' Opposition lineup ': 'exit',\
                    ' \n'              : 'exit'}
        
        try:
            return functions[text]
        except KeyError:
            return 'continue'
        
    def read_ids(self, file):
        self.ids = {}
        with open(file, 'r') as id_file:
            for line in id_file.readlines():
                if not ('\\N') in line:
                    parts = line.strip().split('\t', 1)
                    self.ids[parts[1]] = parts[0]

    def read_stadiums(self, file):
        self.stadiums = {}
        with open(file, 'r', encoding = 'utf-8') as id_file:
            for line in id_file.readlines():
                parts = line.strip().split('\t', 1)
                #print (parts)
                self.stadiums [parts[1]] = parts[0]

    def read_players(self, file):
        self.players = {}
        with open(file, 'r', encoding = 'utf-8') as id_file:
            for line in id_file.readlines():
                parts = line.strip().split('\t', 2)
                self.players [parts[1]] = parts[0]
        
    def read_edited_players(self, file):
        self.edited_players = {}
        with open(file, 'r', encoding = 'utf-8') as id_file:
            for line in id_file.readlines():
                parts = line.strip().split(', ')
                assert len(parts) >= 4, parts
                
                name    = parts[1].strip()
                l_name  = parts[2].strip()
                id      = parts[3].strip()
                
                assert id.isdigit(), id
                self.edited_players [name ] = [id, l_name]
                        
    def read_own_scorers(self, file):
        self.own_scorers = {}
        with open(file, 'r', encoding = 'utf-8') as id_file:
            for line in id_file.readlines():
                parts = line.strip().split('\t', 2)
                self.own_scorers [parts[0]] = parts[1]
                
    def read_opposition(self, file):
        self.opposition = {}
        with open(file, 'r', encoding = 'utf-8') as id_file:
            for line in id_file.readlines():
                parts = line.strip().split('\t', 1)
                self.opposition[parts[1]] = parts[0]

    def read_referees(self, file):
        self.referees = {}
        with open(file, 'r', encoding = 'utf-8') as id_file:
            for line in id_file.readlines():
                parts = line.strip().split('\t', 1)
                self.referees[parts[1]] = parts[0]

    def read_tournaments(self):
        self.tourn_dict = { 'European Cup Winners':    4653,\
                            '1st Division':            4654,\
                            '2nd Division':            4655,\
                            'Centenary Trophy':        5195,\
                            'Champions League':        310,\
                            'Champions L.':        310,\
                            'CL ':                  310,\
                            'Charity Shield':         4672,\
                            'Community Shield':        4673,\
                            'Europa League':        3243,\
                            'European Cup':            4675,\
                            'European Fairs Cup':    4676,\
                            'European Super Cup':    4677,\
                            'FA Cup'            :    154,\
                            'Lancashire League':    4678,\
                            'League Cup':            319,\
                            'Premier League':        69,\
                            'Screen Sport':            4679,\
                            'Sheriff of London':    4674,\
                            'Test Match':            4680,\
                            'UEFA Cup':                4682,\
                            'World Club':            4681
                         }
                
    def parse_id(self, line):
        self.dict['id'] = self.ids[line]
    
    def parse_opposition(self, name):
        
        try:
            key = self.opposition[name.strip()]
            return key
        except KeyError:
            for key in self.opposition.keys():
                if key.startswith(name.strip()):
                    return self.opposition[key]
            else:
                print (name)
                assert False, name
                return name
        
    def parse_score(self, line):
        pattern = re.compile(r'([\w\s\'-]*)\s(\d{1,2}) - (\d{1,2})\s([\w\s]*)$') 
        #pattern = re.compile(r'[\'\"]([\w\s\'-]*)\s(\d{1,2}) - (\d{1,2})\s([\w\s]*)[\'\"]$') 
            
        try:
            parsed = pattern.search(line).groups()
        except:
            print (line)
            raise ValueError("Wrong Score 1")
        
        if parsed[0] == "Liverpool":
            we_score = parsed[1]
            they_score = parsed[2]
            opposition = parsed[3]
        elif parsed[3] == "Liverpool":
            opposition = parsed[0]
            they_score = parsed[1]
            we_score = parsed[2]
        else: 
            raise ValueError("Wrong Score 2")
            
        self.dict['opposition'] = self.parse_opposition(opposition)
        self.dict['score'] = "{0}:{1}".format(we_score, they_score) 

    def parse_date(self, line):
        date_parts = line.split('.', 2)
        self.dict['date'] = '{0}-{1}-{2}:T00:00:00'.format(date_parts[2],date_parts[1],date_parts[0])

    def parse_stadium(self, line):
        try:
            self.dict['stadium'] = self.stadiums[line]
        except KeyError:
            self.dict['stadium'] = line 
            assert False, line

    def parse_tournament(self, line):
        for key in self.tourn_dict:
            if key in line:
                self.dict['tournament'] = self.tourn_dict[key]
                break
            elif line.strip().find('Eur. Cup') == 0:
                 self.dict['tournament'] = self.tourn_dict['European Cup']
                 break
            elif line.strip().find('SSSC') == 0:
                 self.dict['tournament'] = self.tourn_dict['Screen Sport']
                 break
            elif line.strip().find('Centenary') == 0:
                 self.dict['tournament'] = self.tourn_dict['Centenary Trophy']
                 break
            elif line.strip().find('ECW Cup') == 0:
                 self.dict['tournament'] = self.tourn_dict['European Cup Winners']
                 break
            elif line.strip().find('EFC ') == 0:
                 self.dict['tournament'] = self.tourn_dict['European Fairs Cup']
                 break
            elif line.strip().find('WCC ') == 0:
                 self.dict['tournament'] = self.tourn_dict['World Club']
                 break

        else:
            print ('Not found tournament', line)
            assert False, line

    def parse_attendance(self, line):
       self.dict['attendance'] = line.replace(',', '')
       
    def parse_referee(self, line):
        try:
            self.dict['referee'] = self.referees[line.strip()]
        except KeyError:
            name    = line.strip().split(' ')[0]
            initial = line.strip().split(' ')[1]
            for key in self.referees.keys():
                if key == 'Unknown':
                    continue

                k_name      = key.split(' ')[1]
                k_initial   = key.split(' ')[0][0]
                
                if name ==  k_name and initial  == k_initial:
                    #print ('__Assume {0} is {1}'.format(line, key))
                    self.dict['referee'] = self.referees[key]
                    break

    def parse_player(self, name):
        try: 
            id = self.players[name.strip()]
        except KeyError:
            id = self.edited_players[name.strip()][0]
            print ('__Assume {0} is {1}'.format(name, self.edited_players[name.strip()][1]))
            
        return id                         
    
    def parse_line_up(self, line):
        record = []
        for i in range (0, 11):
            record.append([i+1, line[i*3].strip(), self.parse_player(line[i*3+1].strip() + ' ' + line[i*3+2].strip()), 0, 'NULL'])
        
        self.dict['line-up'] = record

    def parse_sub(self, line):
        self.dict['line-up'].append([len(self.dict['line-up'])+1, line[0].strip(), self.parse_player(line[1].strip() + ' ' + line[2].strip()), 'NULL', 'NULL'])
        return 3
        
    def parse_notes(self, line):
        if  not (line[0]) == ' \n':
            self.dict['notes'] = line

        return 1
    
    def parse_goal(self, line):
        shift = 1
        
        if line[0].strip() in ['\n', 'Substitutions', 'Red Cards']:
            ''' no goals '''
            return 0
                
        if line[0].find('O/G') > 0:
            ''' own goal '''
            scorer =  self.own_scorers[line[0].split('O/G')[0].strip()]
            minute =  line[0].split('O/G')[1].strip()[:-1]
            type = goal_type.own_goal
            if not minute.isdigit():
                minute = 0
                
            self.dict['goals'].append([scorer, type, minute])
            return shift
        else:
            scorer = self.parse_player(line[0].strip())
            if line[1].find(' pen ') > 0:
                type = goal_type.penalty
                minute = line[1].strip().split(' ')[0][:-1]
                shift = 2
            else:
                type = goal_type.ordinary
                minute = line[1].strip()[:-1]

            if minute.isdigit():
                shift = 2
            else:
                ''' no timestamp '''
                minute = 0
                
            self.dict['goals'].append([scorer, type, minute])
            return shift
               
    def parse_red_cards(self, line):
        player = self.parse_player(line[0].strip())
        minute = line[1].strip()[:-1]
        
        if minute.isdigit():
            shift = 2
        else:
            ''' no timestamp '''
            minute = 0
            shift = 1 

        self.dict['red cards'].append([player, minute])
        return shift

    def parse_substitutions(self, line):
        coming_off = line[::4]
        coming_on =  line[2::4]
        time = line[3::4]
        
        assert (line.index(' Opposition lineup ') > 0)
        num_subs = int(line.index(' Opposition lineup ')/4)
        
        for i in range(0, num_subs):
            player_in = self.parse_player(coming_on[i])
            player_out = self.parse_player(coming_off[i])
            
            minute = time[i][2:-2]
            if not minute.isdigit():
                assert minute == ''
                minute = 0
                
            for item in self.dict['line-up']:
                if item[2] == player_in:
                    ''' set came off '''
                    item[3] = minute
                if item[2] == player_out:
                    ''' set came on '''
                    item[4] = minute
               
        ''' skipping space and minute sign '''
        return num_subs*4

    def generate_goals_stats_sql(self, out_file = ''):
        
        if len(self.dict['goals']) == 0:
            return
        
        command = 'INSERT INTO goals_stats (goals_stats.match, goalscorer_id, minute, goal_type, own_goalscorer_id, assistant_id) VALUES '

        #print ('\n{1} goals in match {0}'.format(self.dict['id'], len(self.dict['goals'])))
        match_id = self.dict['id']

        for [scorer, type, minute] in self.dict['goals']:
            assert type >= 0 and type <= 2, type

            if type == goal_type.own_goal:
                goalscorer_id = 'NULL'
                own_goalscorer_id = scorer
            else:
                goalscorer_id = scorer
                own_goalscorer_id = 'NULL'
    
            data = '(\'{0}\', \'{1}\',\'{2}\',\'{3}\',\'{4}\',\'{5}\'),'.format(match_id, goalscorer_id, minute, type, own_goalscorer_id, 'NULL')
            
            command = command + data
        
        command = command[:-1] + ';'
        if out_file == '':
            print (command)
        else:
            out_file.write(command)
        
    def generate_player_stats_sql(self, out_file = ''):
        
        command = 'INSERT INTO player_stats (player_stats.match, player_teamsheet_number, player_shirtnumber, player, came_on, came_off) VALUES '
        
        for data in self.dict['line-up']:
            ''' match id, player teamsheet number, shirt number, player id, came on,
            came off, captain, yellow card, second yellow, red card  '''
            values = '(\'{0}\', \'{1}\',\'{2}\',\'{3}\',\'{4}\',\'{5}\')'\
                          .format(self.dict['id'], data[0], data[1], data[2], data[3], data[4])
                          
            command = command + values + ','
        
        command = command[:-1] + ';\n'

        if out_file == '':
            print (command)
        else:
            out_file.write(command)

    def print_referee_list(self, file):
        all_ref = []
        x = {}
        y = {}
        
        with open(file, 'r', encoding = 'utf-8') as recent_file:
            for line in recent_file.readlines():
                parts = line.split('@')
                try:
                    pos = parts.index(' Referee: ') + 1
                    ref_name = parts[pos].strip()
                    try:
                        self.referees[ref_name]
                        x[ref_name] = self.referees[ref_name]
                    except:
                        ''' search manually '''
                        ref_surname = ref_name.split(' ')[0]
                         
                        for item in self.referees:
                            if ref_surname == item.split(' ')[-1]:
                                #print ('Found {0} in {1}'.format(ref_name, item))
                                x[ref_name] = self.referees[item]
                                break
                        else:
                            all_ref.append(parts[pos].strip())
                except ValueError:
                    pass

        all_ref =  set(all_ref)              
        print (len(all_ref),all_ref)
        print (len(x),x)

        command = 'INSERT INTO term_data (vid, name, description) VALUES (6, \'{0}\', \'{1}\');'
        
        for ref in all_ref:
            if ref.find('(') > 0:
                ''' international ref '''
                line = ref.split('(')
                s1 = line[0].strip()
                s2 = 'from ' + line[1].split(')')[0]
            else:
                s1 = s2 = ref
                
            print (command.format(s1, s2))
        
        return all_ref


    def add_referee_id(self, file):
        all_ref = []
        x = {}
        y = {}
        
        with open(file, 'r', encoding = 'utf-8') as recent_file:
            for line in recent_file.readlines():
                parts = line.split('@')
                try:
                    pos = parts.index(' Referee: ') + 1
                    ref_name = parts[pos].strip()
                    try:
                        self.referees[ref_name]
                        x[ref_name] = self.referees[ref_name]
                    except:
                        ''' search manually '''
                        ref_surname = ref_name.split(' ')[0]
                         
                        for item in self.referees:
                            if ref_surname == item.split(' ')[-1]:
                                #print ('Found {0} in {1}'.format(ref_name, item))
                                x[ref_name] = self.referees[item]
                                break
                        else:
                            all_ref.append(parts[pos].strip())
                except ValueError:
                    pass

        print (len(x),x)

        with open(file, 'r', encoding = 'utf-8') as recent_file:
            for line in recent_file.readlines():
                #print (line)
                parts = line.split('@')
                try:
                    pos = parts.index(' Referee: ') + 1
                    ref_name = parts[pos].strip()
                    if x[ref_name] == '0':
                        pass
                    else:
                        print ('UPDATE content_type_game SET field_referee_value = \'{0}\' WHERE nid = \'{1}\';'.format(x[ref_name], self.ids[parts[0][:-1]]))
                except:
                    pass

    def add_match_stats(self):
        for item in self.full_dict:
            for goal in item['goals']:
                if goal[1] == goal_type.own_goal:
                    data = [item['id'], 0, goal[0], goal[2], 0]
                elif goal[1] == goal_type.penalty:
                    data = [item['id'], goal[0], 0, goal[2], 1]
                elif goal[1] == goal_type.ordinary:
                    data = [item['id'], goal[0], 0, goal[2], 0]
                else:
                    assert False, goal
                
                #if item['date'].split('-')[0] == '1959' and item['date'].split('-')[1] == '08':
                print ('INSERT INTO lfc_goals_stats VALUES (' +  ''.join('\'{}\','.format(item) for item in data)[:-1] + ');')

with open('a', mode= 'r') as inf:
    all = inf.readlines()
    fix = set(all)
    print (len(fix), len(all))
    
    dict = {}
    
    for line in all:
        if line in dict:
            #if not line.split(',')[3] == '\'0\'':
            print (line.split(',')[0])
        else:
            dict[line] = 1
    


'''
print ('Starting')
parser = Parse_Recent_Games()

#parser.read_files('off2/', 'data/off3.txt')

dump_file = 'data/off2.txt'
parser.process_recent(dump_file)
#print (parser.dict)
#parser.add_referee_id('data/all_official.txt')
parser.add_match_stats()

print ('\nDone')
'''
            
